#!/usr/bin/env node

const cheerio = require('cheerio');
const superagent = require('superagent');
const charset = require('superagent-charset');
const temme = require("temme").default;
const chalk = require('chalk');
const fs = require('fs');
const p = require('path');
const request = charset(superagent);
const request1 = require('request');
const moment = require('moment');
const _ = require('lodash');
const program = require('commander');
const utils = require('./js/utils');

function formatSelector(line) {
    const items = (new Function(`return ${line}`))();
    const group = items.group;
    delete items.group;
    const list = [];
    for (const key in items) {
        const item = items[key];
        const el = item[0] || '&';
        const prop = item[1];
        list.push(prop ? `${el}[${prop}=$${key}];` : `${el}{$${key}};`);
    }
    return group ? `${group}@{${list.join('')}}` : list.join('');
}
function showError(url, info) {
    console.error(chalk.red(url + ':' + info));
    process.exit(0);
}
function parseHtml(html, selector, list) {
    if (list) {
        return temme(html, formatSelector(list));
    }
    if (_.isFunction(selector)) {
        const $ = cheerio.load(html);
        return elector($, { _, moment, utils });
    }
    return temme(html, selector);
}
function parseClip(selector, list) {
    return new Promise((resolve) => {
        utils.paste(text => {
            resolve(parseHtml(text, selector, list));
        });
    });
}
function parseUrl(url, selector, list, encoding) {
    return new Promise((resolve) => {
        request.get(url).charset(encoding).end((err, res) => {
            if (err) {
                showError(url, err.status);
            }
            resolve(parseHtml(res.text, selector, list));
        });
    });
}
async function downloadUrl(url, fullname, path) {
    return new Promise(resolve => {
        request1(url).pipe(fs.createWriteStream(p.join(path, fullname))).on('close', function () {
            resolve(true);
        }).on('error', function (e) {
            resolve();
        })
    });
}
function downloadItem(node, index, path) {
    let exists = false;
    for (const key in node) {
        const url = node[key];
        if (/^https?:\/\//.test(url)) {
            console.log(`开始下载${url}`);
            const match = url.match(/\.\w+$|\.\w+\?/);
            match && (index = `${key}${index}${match[0].replace('?', '')}`);
            downloadUrl(url, index, path).then(success => {
                success ? console.log(`${url} 下载成功`) : console.log(`${url} 下载失败`);
            });
            exists = true;
        }
    }
    return exists;
}
function downloadList(items, path) {
    let list = items;
    !_.isArray(items) && (list = items ? [items] : []);
    let exists = false;
    for (const i in list) {
        const r = downloadItem(list[i], i, path);
        r && (exists = true);
    }
    if (!exists) console.log('没有可下载的文件');
}
// xspider -u http://localhost:4000/1.html -s "\$=>{let list=[];\$('span').each((o,k)=>list.push(\$(k).html())); return list}"
// xspider -u http://localhost:4000/1.html -s "li@{span[data-color=\$color];span{\$name}}"
// xspider -c -l "{ group:'.item', name: ['a'], url: ['a', 'href']}"
// xspider -c -l "{ group:'.items', url: ['img', 'src'] }" -d .
program
    .version('0.0.1')
    .option('-u, --url [http://127.0.0.1:3000/test]', 'url of api')
    .option('-t, --html [<div>xx</div>]', 'html')
    .option('-c, --clip', 'html from clip')
    .option(`-l, --list { group, key: [el, prop]}', 'list和selector二选一，list用来爬取数组数据，group为每一项元素(为空时只有一项数据), 每一项key为结果键值，el为dom元素，prop为属性(空时为值), 例: xspider -c -l "{ group:'.item', name: ['a'], url: ['a', 'href']}"`)
    .option(`-s, --selector [div{$}]', '字符串或者函数如：($, { _, moment, utils })=>..., 使用cheerio解析, 例: xspider -t '' -s "li@{span[data-color=\$color];span{\$name}}  | xspider -t '' -s "\$=>{let list=[];\$('span').each((o,k)=>list.push(\$(k).html())); return list}"`)
    .option('-f, --file [file]', 'use file for selector')
    .option('-e, --encoding [utf-8]', 'utf-8|GBK|gb2312')
    .option('-p, --copy [0|1|2|3]', 'copy to clipboard with format 0: js 1: json 2:inline-js 3:inline-json')
    .option('-o, --out [file]', 'write to dist file with js format')
    .option('-d, --download [path]', '如果是https开头的，下载文件到相应目录')
    .option('selector help: https://temme.js.org')
    .option('获取class为content的div下所有p的内容：.content>p@{&{\\$};}')
    .option('获取class为content的html的内容：.content{html(\\$)}')
    .option('获取.item-list下多个.item的a标签的内容：.item-list>.item@list{a{\\$text};}')
    .option('获取.item-list下多个.item的a标签的href：.item-list>.item@list{a[href=\\$text];}')
    .option('获取.item-list下多个.item的a标签的href和内容：.item-list>.item@list{a{\\$text}[href=\\$href];}')
    .option('多次获取：.item-list>.item@list{a{\\$text}};.item-list>.item@list1{a[href=\\$href];}')
    .parse(process.argv);
async function main() {
    let { url, html, list, clip, copy, selector, file, out, download, encoding } = program;

    const func = file ? fs.readFileSync(file, 'utf-8') : /=>/.test(selector) ? (new Function(`return ${selector}`))() : selector;
    const items = clip ? await parseClip(func, list) : html ? parseHtml(html, func, list) : url ? await parseUrl(url, func, list, encoding) : [];
    console.log(items);
    if (copy !== undefined) {
        copy === true && (copy = 0);
        utils.copy(_.isString(items) ? items : utils.stringify(items, copy));
    } else if (out !== undefined) {
        fs.writeFileSync(out, _.isString(items) ? items : utils.stringify(items, 0));
    }
    if (download) {
        downloadList(items, download);
    }
}
main();
